#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: xaoyaoyao
@contact: xaoyaoyao@aliyun.com
@file: ifeng_spider.py
@time: 2018/08/25
"""

import random

from scrapy.http import Request
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from article_spider.items import ArticleItem

class IfengSpider(CrawlSpider):
    name = 'ifeng'
    allowed_domains = ['ifeng.com']
    start_urls = [
        'http://tech.ifeng.com/listpage/803/1/list.shtml',
    ]

    def parse(self, response):
        current_url = response.url
        self.logger.info('This is an item page! %s', current_url)
        try:
            next_link = LinkExtractor(restrict_xpaths='//div[@class="m_page"]/a[@id="pagenext"]')
            if next_link:
                n_link = next_link.extract_links(response)
                if n_link:
                    next_url = n_link[0].url
                    self.logger.info('The next item page! %s', next_url)
                    yield Request(url=next_url, callback=self.parse)
        except Exception as e:
            self.logger.error("the next link error. The msg %s", str(e))
        try:
            d_link = LinkExtractor(restrict_xpaths='//div[@id="box_content"]/div[contains(@class,"zheng_list")]//a[@class="t_css"]')
            if d_link:
                links = d_link.extract_links(response)
                if links:
                    for detail_link in links:
                        if detail_link:
                            yield Request(url=detail_link.url, callback=self.parse_detail)
        except Exception as e:
            self.logger.error("the link detail error. The msg %s", str(e))


    def parse_detail(self, response):
        article_item = ArticleItem()
        current_url = response.url
        self.logger.info('This is an item page! %s', current_url)
        article_item['url'] = current_url
        # title
        rt_title = response.css('#artical #artical_topic::text').extract()
        title = ''
        if rt_title and len(rt_title) > 0:
            title = rt_title[0]
        article_item['title'] = title
        # content
        rt_content = response.css('#main_content p::text').extract()
        content = ''
        if rt_content:
            for c_text in rt_content:
                content += str(c_text) + '\n'
                content = content.replace('　', '')
        article_item['content'] = content
        rt_date = response.css('#artical p.p_time span.ss01::text').extract()
        _date = None
        if rt_date:
            if rt_date and len(rt_date) > 0:
                _date = rt_date[0]
        rt_type = response.css('div.theLogo div.theCurrent.cDGray.js_crumb a::text').extract()
        _type = ''
        if rt_type and len(rt_type) > 0:
            _type = rt_type[0]
        article_item['type'] = _type
        article_item['name'] = '凤凰网'
        article_item['date'] = _date
        article_item['grade'] = random.randint(88, 95)
        article_item['domain'] = 'ifeng.com'
        self.logger.info('article_item >> %s', article_item)
        yield article_item